{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-01-15T05:38:06.657193Z",
     "start_time": "2025-01-15T05:38:06.545176Z"
    }
   },
   "source": [
    "# 小批量梯度下降法\n",
    "import numpy as np"
   ],
   "outputs": [],
   "execution_count": 2
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "$$\n",
    "y = 4 + 3x + noise\n",
    "$$"
   ],
   "id": "c5d4ae6705810a98"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-15T05:38:06.672192Z",
     "start_time": "2025-01-15T05:38:06.658200Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 创建数据集\n",
    "X = 2 * np.random.rand(100, 1)\n",
    "y = 4 + 3 * X + np.random.randn(100, 1)\n",
    "X_b = np.c_[np.ones((100, 1)), X]"
   ],
   "id": "6a1bf122a6bcbac0",
   "outputs": [],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-15T05:38:06.688191Z",
     "start_time": "2025-01-15T05:38:06.673178Z"
    }
   },
   "cell_type": "code",
   "source": "X_b",
   "id": "c77c893f7fd52d0e",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.00000000e+00, 1.09001778e+00],\n",
       "       [1.00000000e+00, 1.33113016e+00],\n",
       "       [1.00000000e+00, 1.01256119e+00],\n",
       "       [1.00000000e+00, 6.40439965e-01],\n",
       "       [1.00000000e+00, 1.46843457e+00],\n",
       "       [1.00000000e+00, 2.77477646e-01],\n",
       "       [1.00000000e+00, 3.00148194e-02],\n",
       "       [1.00000000e+00, 5.47828719e-01],\n",
       "       [1.00000000e+00, 1.39155247e-01],\n",
       "       [1.00000000e+00, 6.69655100e-03],\n",
       "       [1.00000000e+00, 1.84533544e+00],\n",
       "       [1.00000000e+00, 1.21380157e+00],\n",
       "       [1.00000000e+00, 1.14378390e+00],\n",
       "       [1.00000000e+00, 1.04171162e+00],\n",
       "       [1.00000000e+00, 9.81855275e-01],\n",
       "       [1.00000000e+00, 6.13959359e-01],\n",
       "       [1.00000000e+00, 1.29408972e+00],\n",
       "       [1.00000000e+00, 1.43286675e+00],\n",
       "       [1.00000000e+00, 1.65494847e+00],\n",
       "       [1.00000000e+00, 1.40995882e+00],\n",
       "       [1.00000000e+00, 1.03126519e+00],\n",
       "       [1.00000000e+00, 1.40220817e+00],\n",
       "       [1.00000000e+00, 1.87909492e+00],\n",
       "       [1.00000000e+00, 1.04972214e+00],\n",
       "       [1.00000000e+00, 9.66626999e-01],\n",
       "       [1.00000000e+00, 1.91039380e+00],\n",
       "       [1.00000000e+00, 3.08351687e-01],\n",
       "       [1.00000000e+00, 1.89082193e+00],\n",
       "       [1.00000000e+00, 1.99531724e+00],\n",
       "       [1.00000000e+00, 1.47036477e+00],\n",
       "       [1.00000000e+00, 2.48998584e-01],\n",
       "       [1.00000000e+00, 4.26453151e-01],\n",
       "       [1.00000000e+00, 9.37112685e-01],\n",
       "       [1.00000000e+00, 1.25560050e-01],\n",
       "       [1.00000000e+00, 1.87416320e+00],\n",
       "       [1.00000000e+00, 1.46301042e+00],\n",
       "       [1.00000000e+00, 1.95425855e+00],\n",
       "       [1.00000000e+00, 1.98971644e+00],\n",
       "       [1.00000000e+00, 7.97554460e-02],\n",
       "       [1.00000000e+00, 2.94331231e-01],\n",
       "       [1.00000000e+00, 7.11786735e-01],\n",
       "       [1.00000000e+00, 1.51427263e+00],\n",
       "       [1.00000000e+00, 9.10177159e-01],\n",
       "       [1.00000000e+00, 1.90190888e+00],\n",
       "       [1.00000000e+00, 1.34198402e+00],\n",
       "       [1.00000000e+00, 2.13591713e-01],\n",
       "       [1.00000000e+00, 9.29537583e-01],\n",
       "       [1.00000000e+00, 1.15321093e-01],\n",
       "       [1.00000000e+00, 1.63464291e+00],\n",
       "       [1.00000000e+00, 1.17614369e+00],\n",
       "       [1.00000000e+00, 9.05366384e-01],\n",
       "       [1.00000000e+00, 1.53808613e+00],\n",
       "       [1.00000000e+00, 3.20908384e-01],\n",
       "       [1.00000000e+00, 8.98960948e-02],\n",
       "       [1.00000000e+00, 7.09806893e-01],\n",
       "       [1.00000000e+00, 8.55296031e-01],\n",
       "       [1.00000000e+00, 2.93666470e-01],\n",
       "       [1.00000000e+00, 5.82868930e-01],\n",
       "       [1.00000000e+00, 1.55673271e+00],\n",
       "       [1.00000000e+00, 1.52959005e+00],\n",
       "       [1.00000000e+00, 8.14581073e-01],\n",
       "       [1.00000000e+00, 1.03883064e+00],\n",
       "       [1.00000000e+00, 1.64421938e+00],\n",
       "       [1.00000000e+00, 1.82509837e+00],\n",
       "       [1.00000000e+00, 4.90511346e-01],\n",
       "       [1.00000000e+00, 9.33931682e-01],\n",
       "       [1.00000000e+00, 1.63314395e+00],\n",
       "       [1.00000000e+00, 3.67504411e-01],\n",
       "       [1.00000000e+00, 7.71991367e-03],\n",
       "       [1.00000000e+00, 1.13678392e+00],\n",
       "       [1.00000000e+00, 4.57365735e-01],\n",
       "       [1.00000000e+00, 7.98995298e-01],\n",
       "       [1.00000000e+00, 1.98631346e+00],\n",
       "       [1.00000000e+00, 1.97360715e-03],\n",
       "       [1.00000000e+00, 1.56603160e+00],\n",
       "       [1.00000000e+00, 1.79572217e+00],\n",
       "       [1.00000000e+00, 1.35902133e+00],\n",
       "       [1.00000000e+00, 1.65417641e-01],\n",
       "       [1.00000000e+00, 1.84457891e+00],\n",
       "       [1.00000000e+00, 1.69910890e+00],\n",
       "       [1.00000000e+00, 7.96065193e-02],\n",
       "       [1.00000000e+00, 3.84131145e-01],\n",
       "       [1.00000000e+00, 1.82914429e+00],\n",
       "       [1.00000000e+00, 6.73443880e-01],\n",
       "       [1.00000000e+00, 1.68696557e+00],\n",
       "       [1.00000000e+00, 1.98238331e+00],\n",
       "       [1.00000000e+00, 2.75695458e-01],\n",
       "       [1.00000000e+00, 7.66816407e-03],\n",
       "       [1.00000000e+00, 1.32330090e+00],\n",
       "       [1.00000000e+00, 3.01183105e-01],\n",
       "       [1.00000000e+00, 1.10531301e+00],\n",
       "       [1.00000000e+00, 1.63576598e+00],\n",
       "       [1.00000000e+00, 2.71906969e-01],\n",
       "       [1.00000000e+00, 8.34866242e-01],\n",
       "       [1.00000000e+00, 8.10286945e-01],\n",
       "       [1.00000000e+00, 7.19927464e-02],\n",
       "       [1.00000000e+00, 1.16969063e+00],\n",
       "       [1.00000000e+00, 9.95878470e-02],\n",
       "       [1.00000000e+00, 1.82973948e+00],\n",
       "       [1.00000000e+00, 1.51902362e+00]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-15T05:38:06.703191Z",
     "start_time": "2025-01-15T05:38:06.690177Z"
    }
   },
   "cell_type": "code",
   "source": "y",
   "id": "acf9f0ad2ef72835",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 7.87983965],\n",
       "       [ 8.25563889],\n",
       "       [ 6.54047379],\n",
       "       [ 5.15616527],\n",
       "       [ 8.96348969],\n",
       "       [ 4.98175273],\n",
       "       [ 5.69614459],\n",
       "       [ 5.26123069],\n",
       "       [ 5.55956545],\n",
       "       [ 4.58723951],\n",
       "       [10.68738219],\n",
       "       [ 8.92706129],\n",
       "       [ 9.26903912],\n",
       "       [ 6.35493183],\n",
       "       [ 5.10761186],\n",
       "       [ 6.44719978],\n",
       "       [ 8.43680744],\n",
       "       [ 7.06210981],\n",
       "       [ 9.38466082],\n",
       "       [ 7.40696711],\n",
       "       [ 8.42702429],\n",
       "       [ 8.00856251],\n",
       "       [ 8.12930376],\n",
       "       [ 7.24588356],\n",
       "       [ 7.57229027],\n",
       "       [10.4620761 ],\n",
       "       [ 4.82591249],\n",
       "       [ 9.32376306],\n",
       "       [ 9.65948654],\n",
       "       [ 7.65349688],\n",
       "       [ 4.61757871],\n",
       "       [ 4.81559932],\n",
       "       [ 7.64793658],\n",
       "       [ 5.63427715],\n",
       "       [ 9.66292873],\n",
       "       [ 7.87304224],\n",
       "       [ 9.02188857],\n",
       "       [ 9.57659676],\n",
       "       [ 4.42183911],\n",
       "       [ 4.96845195],\n",
       "       [ 5.57631598],\n",
       "       [ 9.59459976],\n",
       "       [ 6.98695014],\n",
       "       [11.26578375],\n",
       "       [ 8.66577495],\n",
       "       [ 5.12781213],\n",
       "       [ 5.40574139],\n",
       "       [ 2.59968691],\n",
       "       [ 6.94551617],\n",
       "       [ 8.44317856],\n",
       "       [ 5.98633717],\n",
       "       [ 7.42027276],\n",
       "       [ 4.31812932],\n",
       "       [ 6.20326998],\n",
       "       [ 3.82812054],\n",
       "       [ 5.38599707],\n",
       "       [ 4.79081998],\n",
       "       [ 6.76206493],\n",
       "       [ 9.06385734],\n",
       "       [ 9.72134987],\n",
       "       [ 6.72587103],\n",
       "       [ 6.60836525],\n",
       "       [ 9.99335636],\n",
       "       [10.68651876],\n",
       "       [ 4.98643364],\n",
       "       [ 7.27488042],\n",
       "       [ 9.00520144],\n",
       "       [ 4.18262872],\n",
       "       [ 2.30916224],\n",
       "       [ 6.8196683 ],\n",
       "       [ 5.01312229],\n",
       "       [ 6.35887738],\n",
       "       [ 9.88034287],\n",
       "       [ 4.97186515],\n",
       "       [ 8.03685205],\n",
       "       [ 8.59532778],\n",
       "       [ 8.97156998],\n",
       "       [ 2.93159106],\n",
       "       [ 9.94340665],\n",
       "       [ 7.22024102],\n",
       "       [ 4.03193699],\n",
       "       [ 3.58515502],\n",
       "       [ 9.37979388],\n",
       "       [ 6.65165337],\n",
       "       [ 8.23897423],\n",
       "       [11.95329346],\n",
       "       [ 2.41900047],\n",
       "       [ 4.38924974],\n",
       "       [ 8.59283649],\n",
       "       [ 4.0606302 ],\n",
       "       [ 6.33577115],\n",
       "       [ 8.3120471 ],\n",
       "       [ 5.82526807],\n",
       "       [ 5.73891743],\n",
       "       [ 5.46337252],\n",
       "       [ 3.26775966],\n",
       "       [ 8.04037397],\n",
       "       [ 2.99984903],\n",
       "       [ 9.12266641],\n",
       "       [ 9.23828639]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-15T05:38:06.719198Z",
     "start_time": "2025-01-15T05:38:06.705176Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 设置超参数\n",
    "learning_rate = 0.001  # 学习率\n",
    "n_epochs = 10000  # 迭代次数\n",
    "m = len(X_b)  # 样本数量\n",
    "batch_size = 10  # 批量大小\n",
    "\n",
    "num_batches = int(np.ceil(m / batch_size))  # 计算批量数量"
   ],
   "id": "ea505d9868bd8c22",
   "outputs": [],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-15T05:38:06.735236Z",
     "start_time": "2025-01-15T05:38:06.721176Z"
    }
   },
   "cell_type": "code",
   "source": "# 初始化 theta权重系数：w0,w1,w2,....,wn，正态分布创建 W",
   "id": "932318ed2f7356ad",
   "outputs": [],
   "execution_count": 7
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-15T05:38:06.750759Z",
     "start_time": "2025-01-15T05:38:06.736758Z"
    }
   },
   "cell_type": "code",
   "source": [
    "theta = np.random.randn(2, 1)\n",
    "theta"
   ],
   "id": "bfd19bd5593e742c",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.75432358],\n",
       "       [-0.0873641 ]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 8
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "$$\n",
    "\\theta_j^{t+1}=\\theta_j^t-\\eta\\cdot(h_\\theta(x^{(i)})-y^{(i)})\\cdot x_j^{(i)} \\\\\n",
    "h_w(x) = \\theta^T X  \\\\\n",
    "gradient_i=(\\theta^T X-y)\\cdot x_j \\\\\n",
    "gradient_i=(X\\theta-y)\\cdot x_j \\\\\n",
    "gradient_i= X^T \\cdot (X\\theta-y)\\\\\n",
    "$$"
   ],
   "id": "c7823524eeac3a2"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-15T05:38:07.376778Z",
     "start_time": "2025-01-15T05:38:06.752758Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 和之前代码差别无非就是切片取出来的是 batch_size 个样本数据，然后求解梯度而已\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    for i in range(num_batches):\n",
    "        # 求gradient\n",
    "        random_index = np.random.randint(m)\n",
    "\n",
    "        x_batch = X_b[random_index:random_index + batch_size]\n",
    "        y_batch = y[random_index:random_index + batch_size]\n",
    "        gradients = x_batch.T.dot(x_batch.dot(theta) - y_batch)\n",
    "\n",
    "        # 更新theta\n",
    "        theta = theta - learning_rate * gradients\n",
    "\n",
    "theta"
   ],
   "id": "c40be957a0ec6e55",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.74248322],\n",
       "       [3.13410371]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 9
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-15T05:40:06.540321Z",
     "start_time": "2025-01-15T05:40:06.111072Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 思考题 1\n",
    "# 这里的随机梯度下降和小批量梯度下降的随机方式，会不会有可能有些数据一直都取不到的可能？代码如何操作解决这个问题？\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    # 在双层 for 循环之间，也就是每个轮次开始分批次迭代之前打乱数据索引顺序\n",
    "    arr = np.arange(len(X))\n",
    "    np.random.shuffle(arr)\n",
    "    X_b = X_b[arr]\n",
    "    y = y[arr]\n",
    "    for i in range(num_batches):\n",
    "        # 求gradient\n",
    "        random_index = np.random.randint(m)\n",
    "\n",
    "        x_batch = X_b[random_index:random_index + batch_size]\n",
    "        y_batch = y[random_index:random_index + batch_size]\n",
    "        gradients = x_batch.T.dot(x_batch.dot(theta) - y_batch)\n",
    "\n",
    "        # 更新theta\n",
    "        theta = theta - learning_rate * gradients\n",
    "\n",
    "theta"
   ],
   "id": "bb2f530428ca56f8",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.81828531],\n",
       "       [3.13207792]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 16
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-15T06:34:50.653130Z",
     "start_time": "2025-01-15T06:34:50.639628Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 思考题 2\n",
    "# 如何可以通过代码让学习率随着迭代次数增多而逐渐变小？\n",
    "# 初始学习率等于 5, 500\n",
    "\n",
    "t0, t1 = 5, 500  # 超参数\n",
    "\n",
    "\n",
    "# 定义调整学习率的函数\n",
    "def learning_schedule(t):\n",
    "    return t0 / (t + t1)"
   ],
   "id": "61299e6a1bea6ebd",
   "outputs": [],
   "execution_count": 40
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-15T06:34:51.063290Z",
     "start_time": "2025-01-15T06:34:51.052284Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 在更新 theta 参数前随着 epoch 调整学习率\n",
    "learning_rate = learning_schedule(epoch * m + i)\n",
    "learning_rate"
   ],
   "id": "3f1ae7f68411c255",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.99795583606305e-06"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 41
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-15T06:34:52.060840Z",
     "start_time": "2025-01-15T06:34:51.514037Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 思考题 2 应用\n",
    "for epoch in range(n_epochs):\n",
    "    # 在双层 for 循环之间，也就是每个轮次开始分批次迭代之前打乱数据索引顺序\n",
    "    arr = np.arange(len(X))\n",
    "    np.random.shuffle(arr)\n",
    "    X_b = X_b[arr]\n",
    "    y = y[arr]\n",
    "    for i in range(num_batches):\n",
    "        # 求gradient\n",
    "        random_index = np.random.randint(m)\n",
    "\n",
    "        x_batch = X_b[random_index:random_index + batch_size]\n",
    "        y_batch = y[random_index:random_index + batch_size]\n",
    "        gradients = x_batch.T.dot(x_batch.dot(theta) - y_batch)\n",
    "\n",
    "        # 在更新 theta 参数前随着 epoch 调整学习率\n",
    "        learning_rate = learning_schedule(epoch * m + i)\n",
    "\n",
    "        # 更新theta\n",
    "        theta = theta - learning_rate * gradients\n",
    "\n",
    "theta"
   ],
   "id": "aeb0f97f99d31adb",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.7842503],\n",
       "       [3.1288028]])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 42
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "cd8a2967314b1f67"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
